---------------------------------------------------------------------------------------
-- Private methods which support HTTP communication using NetStreams streams
-- Support module for http.lua
-- Adapted from LuaSocket 1.4 toolkit.
-- Author:  Josh Kewley
-- Company: Insario Corporation
--          Scottsdale, AZ
-- Date: 03/27/2008
---------------------------------------------------------------------------------------
local URL = require "url"

module(...,package.seeall)

USERAGENT = "InsarioAgent 1.0"
BLOCKSIZE = 8192
RevisionNumber="0.63"
ResponseComplete=0

function DB(message)
    debug("Http", makeReadable(message))
end

function DBD(message)
    debug("HttpDetail", makeReadable(message))
end

local function DBS(message)
    debug("HttpStream", makeReadable(message))
end

local function DBK(message)
    debug("HttpSocket", makeReadable(message))
end

function makeReadable(debugString)
  if (debugString==nil) then return "" end
  return debugString:gsub("\n" , "\\n"):gsub("\r" , "\\r")
end


-----------------------------------------------------------------------------
-- Reads from the socket one char at a time until character is found. Return including that character
--   stream: NetStreams stream connected to the server
--   chunkSize: number of bytes to read at a time. Default is 1
--   endChar: read until endchar is found. Defaults to newline character if not provided. Note that this will not work if chunkSize is greater than 1
-- Returns
--   received string
-----------------------------------------------------------------------------
function ReceiveUntilNewLineOrChar(stream, chunkSize, endChar)
    chunkSize = chunkSize or 1
    endChar = endChar or "\n"
    local data = {};
    local aChar, bytesLeft;
    DBS("Reading "..tostring(chunkSize).." chars from stream until "..endChar)

    repeat
      --TODO: Verify that this handles situtations where the character is not found without failing in the case of a slow message arrival
      aChar = ReadChunk(stream, chunkSize);
      bytesLeft = stream:available()
      data[#data+1]=aChar;
    until aChar==endChar or bytesLeft==0

    DBS("DONE. Read "..tostring(#data).." characters : "..table.concat(data))
    return table.concat(data)
end

-----------------------------------------------------------------------------
-- Reads a chunk from the stream. This is the only place where the stream is read from
--   stream: NetStreams stream connected to the server
--   chunkSize: number of bytes to read 
-- Returns
--   received string
-----------------------------------------------------------------------------
function ReadChunk(stream, chunkSize)
    chunkSize = chunkSize or 1
    -- TMI, baby
    --DBS("Reading "..tostring(chunkSize).." chars from stream.")
    return stream:read(chunkSize);
end

----------------------------------------------------------------------------
--- Returns the next line from the socket without the \r\n
-----------------------------------------------------------------------------
function ReadLine(stream)
    local linePlusCRLF = ""
    linePlusCRLF = ReceiveUntilNewLineOrChar(stream)
    return linePlusCRLF:gsub("\r", ""):gsub("\n", "");
end

-----------------------------------------------------------------------------
-- Tries to send data to the server and closes socket on error. This is the only place where the stream is written to
--   stream: NetStreams stream connected to the server
--   data: data to send
-- Returns
--   err: error message if any, nil if successfull
-----------------------------------------------------------------------------
function WriteToStream(stream, data)
    DBS("Sending : ["..data.."]")
    stream:write(data)
    return nil
end

-----------------------------------------------------------------------------
-- Extracts status code from HTTP status line
-- Input
--   line: HTTP status line
-- Returns
--   code: integer with status code, or nil if malformed line
-----------------------------------------------------------------------------
function ParseStatusCode(line)
    DBD("Parse status code from : "..line)
    local code, _ 
    _, _, code = line:find("HTTP/%d*%.%d* (%d%d%d)")
    if(code) then 
      DB("HTTP response status code : "..code) 
      return tonumber(code)
    else
      DB("HTTP response status code was nil") 
      return nil
    end
end

-----------------------------------------------------------------------------
-- Receive server reply messages, parsing for status code
-- Input
--   stream: NetStreams stream connected to the server
-- Returns
--   code: server status code or nil if error
--   line: full HTTP status line
-----------------------------------------------------------------------------
function ReceiveHttpStatus(stream)
    local line, num
    line = ReadLine(stream)
    num = ParseStatusCode(line)
    return num, line
end

-----------------------------------------------------------------------------
-- Receive and parse response header fields
-- Input
--   stream: NetStreams stream connected to the server
--   headers: a table that might already contain headers
-- Returns
--   headers: a table with all headers fields in the form
--        {name_1 = "value_1", name_2 = "value_2" ... name_n = "value_n"}
--        all name_i are lowercase
--   error message in case of error, or nil
-----------------------------------------------------------------------------
function ReceiveHeaders(stream, headers)
    local line, err
    local name, value, _
    -- get first line
    line = ReadLine(stream)
    -- headers go until a blank line is found
    while line ~= "" do
        DBD("parsing raw header [ "..line.."]")

        -- get field-name and value
        _,_, name, value = line:find("^(.-):%s*(.*)")
        if not name or not value then
            DB("malformed response header: "..line)
            return nil, "malformed response header "..line
        end
        name = name:lower()
        
        -- get next line (value might be folded across multiple lines)
        line = ReadLine(stream)
        -- unfold any folded values
        while line:find("^%s+") do
            DBD("continuing to parse folded header")
            value = value .. line
            line = ReadLine(stream)
        end
        -- save pair in table
        DBD("NAME : "..name.." VALUE : "..value)
        if headers[name] then headers[name] = headers[name] .. ", " .. value
        else headers[name] = value end
    end
    DBD("Fetched "..tostring(#headers).." headers.")
    return headers, nil
end

-----------------------------------------------------------------------------
-- Receives a chunked message body
-- Input
--   stream: NetStreams stream connected to the server
--   headers: header set in which to include trailer headers
--   receiveCallback: function to receive chunks
-- Returns
--   nil if successfull or an error message in case of error
-----------------------------------------------------------------------------
function ReceiveBodyInChunks(stream, headers, receiveCallback)
    local chunk, size, line, err, go, uerr, _
    DB("Receive body in chunks")
    while 1 do
        -- get chunk size, skip extention
        line = ReadLine(stream)
        size = tonumber(gsub(line, ";.*", ""), 16)
        if not size then
            err = "invalid chunk size"
            go, uerr = receiveCallback(nil, err)
            return uerr or err
        end
        DBD("Chunk size is "..size)
        
        -- was it the last chunk?
        if size <= 0 then break end
        
        -- get chunk
        chunk, err = ReadChunk(stream, size)
        if err then
            go, uerr = receiveCallback(nil, err)
            return uerr or err
        end
        -- pass chunk to callback
        go, uerr = receiveCallback(chunk)
        if not go then
            return uerr or "aborted by callback"
        end
    end
    
    -- the server should not send trailer headers because we didn't send a
    -- header informing  it we know  how to deal with them. Just in case they do...
    headers, err = ReceiveHeaders(stream, headers)

    -- let callback know we are done
    go, uerr = receiveCallback("")
    return uerr
end

-----------------------------------------------------------------------------
-- Receives a message body by content-length
-- Input
--   stream: NetStreams stream connected to the server
--   length: message body length
--   receiveCallback: function to receive chunks
-- Returns
--   nil if successfull or an error message in case of error
-----------------------------------------------------------------------------
function ReceiveBodyByLength(stream, length, receiveCallback)
    local uerr, go
    DB("Receive body by length : "..tostring(length))
    while length > 0 do
        local size = math.min(BLOCKSIZE, length)
        local chunk = ReadChunk(stream, size)
        go, uerr = receiveCallback(chunk)
        if not go then
            return uerr or "body reading aborted by callback"
        end
        length = length - size
    end
    go, uerr = receiveCallback("")
    return uerr
end

-----------------------------------------------------------------------------
-- Receives a message body until nothing is left on the wire
-- Input
--   stream: NetStreams stream connected to the server
--   receiveCallback: function to receive chunks
-- Returns
--   nil if successfull or an error message in case of error
-----------------------------------------------------------------------------
function ReceiveUntilEmpty(stream, receiveCallback)
    local uerr, go
    DB("Receive body until stream is closed")
    while (stream:available()>0) do
        local size = math.min(BLOCKSIZE, stream:available())
        local chunk = ReadChunk(stream, size)
        receiveCallback(chunk)
    end
    go, uerr = receiveCallback("")
    return uerr
end

-----------------------------------------------------------------------------
-- Receives HTTP response body by passing the data on the wire to a callback function
-- Input
--   stream: NetStreams stream connected to the server
--   headers: response header fields
--   receiveCallback: function to receive chunks
-- Returns
--    nil if successfull or an error message in case of error
-----------------------------------------------------------------------------
function ReceiveBody(stream, headers, receiveCallback)
    local te = headers["transfer-encoding"]
    -- possible transfer encodings: chunked, identity, gzip, compress, deflate. This driver only handles chunked.
    if te then 
      if te == "chunked" then
        return ReceiveBodyInChunks(stream, headers, receiveCallback)
      else
        DB("Response uses an unhandled transfer endcoding "..te..". This Lua driver only handles chunked")
      end 
    elseif tonumber(headers["content-length"]) then
        local length = tonumber(headers["content-length"])
        return ReceiveBodyByLength(stream, length, receiveCallback)
    else
        return ReceiveUntilEmpty(stream, receiveCallback)
    end
end

-----------------------------------------------------------------------------
-- Drop the HTTP response that is in the incoming stream
-- Input
--   stream: NetStreams stream connected to the server
--   headers: response header fields
-- Returns
--    nil if successfull or an error message in case of error
-----------------------------------------------------------------------------
function DumpBodyContents(stream, headers)
    DBD("Dump the body contents")
    return ReceiveBody(stream, headers, function (c, e) return 1 end)
end

-----------------------------------------------------------------------------
-- Sends mime headers
-- Input
--   stream: NetStreams stream with the server
--   headers: table with mime headers to be sent
-- Returns
--   err: error message if any
-----------------------------------------------------------------------------
function WriteHeaders(stream, headers)
    DB("Sending headers")
    headers = headers or {}
    -- send request headers
    for i, v in pairs(headers) do
        DBD(" --> "..i..": "..v)
        WriteToStream(stream, i .. ": " .. v .. "\r\n")
    end
    -- mark end of request headers with an empty newline
    WriteToStream(stream, "\r\n")
end

-----------------------------------------------------------------------------
-- Sends a HTTP request message through NetStreams stream
-- Input
--   stream: NetStreams stream connected to the server
--   request: request table
--   uri: request uri
-- Returns
--   err: nil in case of success, error message otherwise
-----------------------------------------------------------------------------
function SendRequest(stream, request, uri)
    local size, done, err
    DB(request.method.."ing to thermostat")
    WriteToStream(stream, request.method .. " " .. uri .. " HTTP/1.1\r\n")
    
    -- send request headers
    WriteHeaders(stream, request.headers)
    
    -- send request message body, if any
    if request.body then 
      DB("Sending request body")
      WriteToStream(stream, request.body) 
    end
end

-----------------------------------------------------------------------------
-- Determines if we should read a message body from the server response
-- Input
--   request: a table with the original request information
--   response: a table with the server response information
-- Returns
--   1 if a message body should be processed, nil otherwise
-----------------------------------------------------------------------------
function ResponseHasBody(request, response)
    DBD("Checking to see if response has body.  Status code is : "..tostring(response.code))
    if request.method == "HEAD" then return nil end
    if response.code == 204 or response.code == 304 then return nil end
    if response.code >= 100 and response.code < 200 then return nil end
    return 1
end

-----------------------------------------------------------------------------
-- Converts field names to lowercase and adds a few needed headers
-- Input
--   headers: request header fields
--   parsed: parsed request URL
-- Returns
--   lower: a table with the same headers, but with lowercase field names
-----------------------------------------------------------------------------
function BuildRequestHeaders(request, parsed)
    local lowHead = {}
    headers = request.headers or {}

    -- set default headers
    lowHead["user-agent"] = USERAGENT
    DBD("user agent : "..lowHead["user-agent"])
    if request.body then
        size = request.body:len()
        DBD("Adding content-length header: "..tostring(size))
        headers["Content-Length"] = tostring(size)
    end
        
    -- override with user values
    for i,v in pairs(headers) do
        DBD("Add header : "..i:lower().." = "..v)
        lowHead[i:lower()] = v
    end  
    lowHead["host"] = parsed.host
    DBD("host : "..lowHead["host"])
    -- ask the server to automatically close the connection
    lowHead["connection"] = "close"
    DBD("connection : "..lowHead["connection"])
    if(request.method=="POST") then
        DBD("POST, so adding application/x-www-form-urlencoded")
        lowHead["Content-Type"]="application/x-www-form-urlencoded"
    end

    if (parsed.user and parsed.pass) then
        DBD("Add auth headers")
        lowHead["Authorization"] = "Basic " .. URL.base64(parsed.user .. ":" .. parsed.pass)
        DBD(lowHead["Authorization"])
    end

    return lowHead
end

-----------------------------------------------------------------------------
-- Decides wether we should follow retry with authorization formation
-- Input
--   request: a table with the original request information
--   parsed: parsed request URL
--   response: a table with the server response information
-- Returns
--   1 if we should retry, nil otherwise
-----------------------------------------------------------------------------
function ServerSentAuthChallenge(request, parsed, response)
    -- if there has been an authorization attempt, it must have failed
    if request.headers["Authorization"] then return nil end
    -- if we don't have authorization information, we can't retry
    if parsed.user and parsed.pass then return 1
    else return nil end
end

-----------------------------------------------------------------------------
-- Returns the result of retrying a request with authorization information
-- Input
--   request: a table with the original request information
--   parsed: parsed request URL
--   response: a table with the server response information
-- Returns
--   response: result of target redirection
-----------------------------------------------------------------------------
function authorize(request, parsed, response)
    request.headers["Authorization"] = "Basic " .. URL.base64(parsed.user .. ":" .. parsed.pass)
    local authorize = {
        redirects = request.redirects,
        method = request.method,
        url = request.url,
        buildBodyCallback = request.buildBodyCallback,
        headers = request.headers
    }
    return ProcessRequest(authorize, response)
end

-----------------------------------------------------------------------------
-- Decides whether we should follow a server redirect message
-- Input
--   request: a table with the original request information
--   response: a table with the server response information
-- Returns
--   1 if we should redirect, nil otherwise
-----------------------------------------------------------------------------
function ServerSentRedirect(request, response)
    local follow = not request.stay
    follow = follow and (response.code == 301 or response.code == 302)
    follow = follow and (request.method == "GET" or request.method == "HEAD")
    follow = follow and not (request.redirects and request.redirects >= 5)
    DBD("Should follow? "..tostring(follow))
    return follow
end

-----------------------------------------------------------------------------
-- Returns the result of a request following a server redirect message.
-- Input
--   request: a table with the original request information
--   response: a table with the following fields:
--     buildBodyCallback: response method body receive-callback
-- Returns
--   response: result of target redirection
-----------------------------------------------------------------------------
function redirect(request, response)
    local redirects = request.redirects or 0
    redirects = redirects + 1
    DB("Redirecting for "..tostring(redirects).." time")
    local redirect = {
        redirects = redirects,
        method = request.method,
        -- the RFC says the redirect URL has to be absolute, but some servers do not respect that
        url = URL.absolute_url(request.url, response.headers["location"]),
        buildBodyCallback = request.buildBodyCallback,
        headers = request.headers
    }
    DB("Redirecting to "..redirect.url)
    local response = ProcessRequest(redirect, response)
    -- we pass the location header as a clue we tried to redirect
    if response.headers then response.headers.location = redirect.url end
    return response
end

-----------------------------------------------------------------------------
-- Computes the request URI from the parsed request URL
-- Input
--   parsed: parsed URL
-- Returns
--   uri: request URI for parsed URL
-----------------------------------------------------------------------------
function request_uri(parsed)
    local uri = ""
    if parsed.path then uri = uri .. parsed.path end
    if parsed.params then uri = uri .. ";" .. parsed.params end
    if parsed.query then uri = uri .. "?" .. parsed.query end
    if parsed.fragment then uri = uri .. "#" .. parsed.fragment end
    DB("Request URI : "..uri)
    return uri
end

-----------------------------------------------------------------------------
-- Builds a request table from a URL or request table
-- Input
--   url_or_request: target url or request table (a table with the fields:
--     url: the target URL
--     user: account user name
--     pass: account password)
-- Returns
--   request: request table
-----------------------------------------------------------------------------
function BuildRequestTable(data)
    local request = {}
    DB("Type of request is "..type(data))
    if type(data) == "table" then
        for i, v in pairs(data) do
            DBD("Request Table: "..i.." : "..v)
            request[i] = v
        end
    else request.url = data end
    return request
end

-----------------------------------------------------------------------------
-- Parse the response from the request
-- This is the only location where the stream is closed. 
-- This function is guaranteed to close the stream
-----------------------------------------------------------------------------
function ParseResponse(theHttpSocketCxn, parsed, request, response)
  if (theHttpSocketCxn:available() < 1) then 
    response.error="Can't parse response because the port has no data"
    DB(response.error)
    theHttpSocketCxn:close()
    return
  end

  response.code=100
  -- ignore any 100-continue messages
  while response.code == 100 do
      -- get server response message, which should be first header
      response.code, response.status = ReceiveHttpStatus(theHttpSocketCxn)
      if response.code == 100 then
        -- Some servers return a response of 100 accompanied by full headers. strip them and loop until a non-100 header
        DBD("Response code of 100, strip headers and look again")
      end
      
      response.headers, response.error = ReceiveHeaders(theHttpSocketCxn, {})
      if (response.error or response.status == nil) then 
        ResponseComplete = 1
        theHttpSocketCxn:close()
        return
      end
  end

  -- decide what to do based on request and response parameters
  if ServerSentRedirect(request, response) then
      DB("Server told us to redirect")
      DumpBodyContents(theHttpSocketCxn, response.headers)
      theHttpSocketCxn:close()
      redirect(request, response)

  elseif ServerSentAuthChallenge(request, parsed, response) then
      DB("Server asked for authenticaion credentials")
      DumpBodyContents(theHttpSocketCxn, response.headers)
      theHttpSocketCxn:close()
      authorize(request, parsed, response)

  elseif response.code~=200 then
      DB("Request returned a non valid response code of "..tostring(response.code)..". Aborting.")
      ReceiveBody(theHttpSocketCxn, response.headers, response.buildBodyCallback)
      response.error="Invalid response code"
      theHttpSocketCxn:close()
      
  elseif ResponseHasBody(request, response) then
      DB("Response has body, getting it")
      response.error = ReceiveBody(theHttpSocketCxn, response.headers, response.buildBodyCallback)
      theHttpSocketCxn:close()

  else
      DB("Good request, but no body returned")
      theHttpSocketCxn:close()
  end
  ResponseComplete=1
end

-----------------------------------------------------------------------------
-- Sends a HTTP request and retrieves the server reply using callbacks to
-- send the request body and receive the response body
-- Input
--  request: a table with the following fields
--    method: "GET", "PUT", "POST" etc (defaults to "GET")
--    url: target uniform resource locator
--    user, password: authentication information
--    headers: request headers to send, or nil if none
--    body: request message body
--    stay: should we refrain from following a server redirect message?
--    response: a table with the following fields:
--    buildBodyCallback: response method body receive-callback
-- Returns
--  response: a table with the following fields:
--  headers: response header fields received, or nil if failed
--  status: server response full status line, or nil if failed
--  code: server status code, or nil if failed
--  error: error message, or nil if successfull
-----------------------------------------------------------------------------
function ProcessRequest(request, response)
  local parsed = URL.parse_url(request.url)
  if parsed.scheme ~= "http" then
  	response.error = string.format("unsupported scheme '%s'", parsed.scheme)
  	return response
  end
  -- explicit authentication info overrides that given by the URL
  parsed.user = request.user or parsed.user
  parsed.pass = request.pass or parsed.pass
  -- default to GET if not specified
  request.method = request.method or "GET"

  -- fill default headers
  DBD("Filling request headers")
  request.headers = BuildRequestHeaders(request, parsed)

  -- build connection string
  local theConnection = "socket://"..parsed.host
  if (parsed.port) then theConnection = theConnection..":"..parsed.port end

  -- try to connect to server
  DBK("Opening a socket connection with "..theConnection)
  theHttpSocketCxn = createStream(theConnection)

  if(theHttpSocketCxn == nil) then
  	DBK("Unable to establish socket connection to server")
  	ResponseComplete=1
  	-- return so that the :onXXX() methods don't try to index the null connection object
  	return nil  
  else
  	DBK("Socket connection to the server has been established")
    -- send request message
    SendRequest(theHttpSocketCxn, request, request_uri(parsed))
    ParseResponse(theHttpSocketCxn, parsed, request, response)
  end
  
  function theHttpSocketCxn:onOpen()
    -- this doesn't actually fire until you try to write to the socket
    -- even then, it only seems to fire after the socket writing process terminates
  	DB(">>> HTTP Stream Opened "..parsed.host)
  end

  function theHttpSocketCxn:onExpire()
  	DB(">>> HTTP Stream Expired "..parsed.host)
  end

  function theHttpSocketCxn:onClose()
  	DB(">>> HTTP Stream Closed "..parsed.host)
  end
end
